
import sys
import time
import random
import re
import pyautogui

from Config import *
from SpiderUtil import *
from ProxyUtil import *
from RabbitUtil import RabbitUtil
from SeleniumUtil import SeleniumUtil
from SeleniumUtil import CheckDriverExit


#selenium
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
#反爬
import undetected_chromedriver as uc
#BeautifulSoup
from bs4 import BeautifulSoup

from ErrorCode import *

#多线程处理
from threading import Thread
#线程池相关
from concurrent.futures import ThreadPoolExecutor, wait, as_completed

#----------------------------------------解析网页 一个列表页面也就是48个详情----------------------------------------
#解析网页中拍卖房产信息列表 "./html/SFList.html"
def SFListParse(htmlcontent, PropName, Province, City):
    #保存一个临时文件
    # with open(Config["Spider"]["workDir"] + "/Temp.html", "wb") as f:
    #     f.write(htmlcontent);

    #开始解析
    soup = BeautifulSoup(htmlcontent, "lxml");

    #解析出来的信息
    SFInfoList = [];

    #获得拍卖房产信息
    SFList = soup.select('div[class="sf-item-list"] li');

    #逐个信息解析
    for Idx,SF in enumerate(SFList):
        try:
            #详细信息页链接 //sf-item.taobao.com/sf_item/712418425380.htm?track_id=405a29ac-7e74-4283-8675-59b884f033ea
            href = SF.select_one("a")['href'];
            href = re.findall(r"^(.*)\?", href)[0];
            code = re.findall(r"(\d+)\.htm", href)[0];

            #-------------------header-section
            #缩略图地址
            imginfos = SF.select_one('a > div.header-section > img');
            img = imginfos['src'];
            #如果是特殊页面，需要进一步获取
            if img=="//g.alicdn.com/s.gif":
                img = imginfos['data-ks-lazyload'];

            #房屋名称
            title = SF.select_one('a > div.header-section > p').text.strip();
            #-------------------info-section
            #如果拍卖已结束，不再进行解析
            #开始时间
            StartTimeStr = SF.select_one('a > div.info-section > p.time.time-todo > span.value').text;
            #预计结束时间
            AboutEndTimeStr = SF.select_one('a > div.info-section > p.time.time-doing > span.value').text;
            #结束时间
            EndTimeStr   = SF.select_one('a > div.info-section > p.time.time-done > span.value').text;

            #价格等信息
            PriceInfo1 = re.sub("[\s]{1,}", ' ', SF.select_one('a > div.info-section > p.price.price-todo.pai-status-todo-show').text);
            ItemList = SF.select('a > div.info-section > p.price.price-assess');
            if len(ItemList)==0:
                PriceInfo2 = "";
            else:
                PriceInfo2 = re.sub("[\s]{1,}", ' ', ItemList[0].text);
            StartTime  = re.sub("[\s]{1,}", ' ', SF.select_one('a > div.info-section > p.time.time-todo.pai-status-todo-show').text);
            #-------------------footer-section
            #关注信息
            GZInfo = re.sub("[\s]{1,}", ' ', SF.select_one('a > div.footer-section').text);

            Status = StatusDictAli(SF["class"][1])
            #输出数据列表
            resultTuple = {
                "houseCode": code,
                "auctionUrl": "https:" + href,
                "banner": "https:" + img,
                "title": title,
                # "startPrice": PriceInfo1.strip(),
                # "estimatePrice": PriceInfo2.strip(),
                # "startTime": StartTimeStr,
                # "aboutEndTime": AboutEndTimeStr,
                # "endTime": EndTimeStr,
                "houseType": PropName,
                "province": Province,
                "city": City,
                "status": Status,
                "origin": '阿里法拍',
            }
            # resultTuple = (code, "https:"+href, "https:"+img, title, PriceInfo1.strip(), PriceInfo2.strip(), StartTime.strip(), GZInfo.strip(), StartTimeStr, AboutEndTimeStr, EndTimeStr, PropName);
            SFInfoList.append(resultTuple);
            #show for debug
            # print("-"*10, Idx, resultTuple);
            #end for
        except Exception as e:
            print("list parse except info ", e)
        
    #end
    return SFInfoList;
    #end



# 滑块go
def ScrollBarHandle(driver):
    # if slideCount > 10:
    #     slideCount = 0
    #     driver.refresh()

    slider = driver.find_element(By.CSS_SELECTOR, 'span#nc_1_n1z')
    # hkLocal = huakuaiEle.location
    # print (hkLocal)

     # 将鼠标移动到坐标 (x, y)
    # pyautogui.moveTo(hkLocal.get('x'), hkLocal.get('y'))
    pyautogui.moveTo(1260, 1200)

    time.sleep(2)
    # pyautogui.mouseDown()
    pyautogui.mouseDown(button='left')


    pyautogui.dragRel(random.choice(range(320,330)), 0, duration=random.choice(range(3,10))) 
    pyautogui.mouseDown(button='left')
    
    # pyautogui.dragRel(20, 0, duration=1) 
    # pyautogui.dragRel(20, 0, duration=1) 
    # pyautogui.mouseUp(button='left')
    time.sleep(3)
    pyautogui.mouseUp()


def LoginHandle(driver):
    ele = driver.find_element(By.CSS_SELECTOR, 'input#fm-login-id')
    ele.clear()
    ele.send_keys(Config["Spider"]["taobaoAccount"])
    time.sleep(1)
    ele = driver.find_element(By.CSS_SELECTOR, 'input#fm-login-password')
    ele.clear()
    ele.send_keys(Config["Spider"]["taobaoPassword"])
    time.sleep(1)

    login_button = WebDriverWait(driver, 10).until(
        EC.element_to_be_clickable((By.XPATH, '//button[contains(@class, "fm-button") and contains(text(), "登录")]'))
    )
    login_button.click()
    time.sleep(3)

    soup = BeautifulSoup(driver.page_source, "lxml")
    lsbtn = soup.select('button.dialog-btn-feedback');
    if len(lsbtn) > 0:
        lsbtnEle = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.XPATH, '//button[contains(@class, "dialog-btn-feedback") and contains(text(), "保持")]'))
        )
        lsbtnEle.click()
        # driver.find_element(By.CSS_SELECTOR, 'button.dialog-btn-feedback').click()
        time.sleep(8)
         

#根据地市信息，获得各地市资产信息列表
def GetSFList(AreaInfo, City, District, CurrPage, HouseType, driver):

    #延时等待浏览器打开完成
    wait = WebDriverWait(driver, 2);

    itime = 8;

    #循环处理
    while True:
        try:
            CheckDriverExit(driver)

            AliSFURL = "https://sf.taobao.com/item_list.htm";
            #打开资产列表页面
            driver.get(AliSFURL);
            time.sleep(2);

            #如果未进入目标页，则触发异常，等待人工处理
            while driver.current_url != AliSFURL:
                print("Not LoginURL Wait!! LoginURL=", AliSFURL);
                WarningPrompt(Type=EList["E101"]);
                time.sleep(itime);
            

            loginCount = 0
            while loginCount < 10:
                # 登录
                soup = BeautifulSoup(driver.page_source, "lxml")
                loginInput = soup.select('input#fm-login-id');
                if len(loginInput) > 0:
                    LoginHandle(driver)            

                # TODO 拉条或者登录操作
                slideCount = 0
                while slideCount < 13:
                    time.sleep(2)
                    soup = BeautifulSoup(driver.page_source, "lxml")
                    sbBtn = soup.select('div.errloading')
                    huakuai = soup.select('span#nc_1_n1z');

                    if len(sbBtn) > 0:
                        driver.find_element(By.CSS_SELECTOR, 'div.errloading').click()
                        time.sleep(1)
                    soup = BeautifulSoup(driver.page_source, "lxml")
                    huakuai = soup.select('span#nc_1_n1z');
                    
                    # 出现了滑块
                    if len(huakuai) > 0:
                        huakuaiEle = driver.find_element(By.CSS_SELECTOR, 'span#nc_1_n1z')
                        if huakuaiEle.is_displayed():
                            ScrollBarHandle(driver)
                            slideCount = slideCount + 1
                            if slideCount % 3 == 0:
                                driver.refresh()
                                time.sleep(5)
                        else:
                            break
                    else: 
                        break

                # 登录
                soup = BeautifulSoup(driver.page_source, "lxml")
                loginInput = soup.select('input#fm-login-id');
                if len(loginInput) > 0:
                    LoginHandle(driver)

                loginCount = loginCount + 1
                
                soup = BeautifulSoup(driver.page_source, "lxml")
                aaaa = soup.select('div.top-nav-container');
                if len(aaaa) > 0:
                    break
            
            #等待页面加载
            time.sleep(itime);
            
            #点击对应省份、地市、资产类别，分别进行列表的获取
            while True:
                try:
                    #如果没有省的信息，则无需处理
                    if AreaInfo[0] == None:
                        print("There is no province.. ", AreaInfo);
                        break;
                    
                    #省份名称
                    provPage = AreaInfo[0].split("?");
                    ProvName = provPage[0];
                    
                    #显示处理的地区
                    print("processing..... ", ProvName);
                    
                    #寻找并点击省份
                    try:
                        #标的物所在地
                        LocationElem = driver.find_element(By.CSS_SELECTOR, 'ul[class*="sf-filter-row-location"]');
                        #省份标签
                        ProvElem = LocationElem.find_element(By.PARTIAL_LINK_TEXT, ProvName);
                        #点击省
                        if ProvElem.is_enabled() and ProvElem.is_displayed():
                            ProvElem.click();
                        #不可点击
                        else:
                            print("Province is not clickable!!!", ProvName);
                            continue;
                    
                        #等待网页加载
                        time.sleep(itime);

                        #确认省份已经被选中
                        ProvElem = driver.find_element(By.CSS_SELECTOR, 'ul.sf-filter-row-location ul.condition > li.unfold > em > a');
                        if not ProvElem.text.startswith(ProvName):
                            print("ProvElem is not be selected!!");
                            continue;

                    except:
                        print("Press Province Error!!!")
                        WarningPrompt(Type=EList["E102"]);
                        #检查浏览器是否可用
                        CheckDriverExit(driver)
                        time.sleep(itime)
                        continue;
                    
                    
                    #------------------------点击地市------------------------
                    #获得地市列表
                    CityElems = driver.find_elements(By.CSS_SELECTOR, 'div[class*="J_SubCondition"] > ul > li');
                    # CityElems = driver.find_elements(By.CSS_SELECTOR, f'div[class*="J_SubCondition"] li.J_FilterCity > em > a');
                    if len(CityElems)==0:
                        print("There is no CityTag!!!");
                        continue;

                    #遍历每个地市，并分别点击
                    for CityIdx in range(len(CityElems)):
                        #地市处理异常时，不断重试本地市
                        Coder = "utf8";
                        while True:
                            try:
                                CityElems = driver.find_elements(By.CSS_SELECTOR, f'div[class*="J_SubCondition"] li.J_FilterCity > em > a');
                                #本次拟点击地市
                                CityElem = CityElems[CityIdx];
                                #城市名称
                                CityName = CityElem.text.split()[0];

                                # 直接到指定城市
                                if City != None and not City.startswith(CityName):
                                    break;
                                # 匹配到一致的了改为None  后面就不判断了
                                City = None;
                                    
                                #点击城市
                                if CityElem.is_enabled() and CityElem.is_displayed():
                                    CityElem.click();
                                #地市不可点击
                                else:
                                    print("City is not clickable!!!", CityElem.text);
                                    continue;
                                
                                #等待页面加载
                                time.sleep(itime);

                                #确认被点击的城市
                                CheckCityElem = driver.find_element(By.CSS_SELECTOR, 'ul.sf-filter-row-location div.J_SubCondition li.J_FilterCity a.selected');
                                if not CheckCityElem.text.startswith(CityName):
                                    print("CityElem is not be selected!!");
                                    continue;

                                # 点城市下的区
                                AreaElems = driver.find_elements(By.CSS_SELECTOR, f'div[class*="small-subcondion"] li > em > a');
                                
                                for AreaIdx in range(len(AreaElems)):
                                    try:
                                        AreaElems = driver.find_elements(By.CSS_SELECTOR, f'div[class*="small-subcondion"] li > em > a');
                                        DistrictName = AreaElems[AreaIdx].text
                                        if District != None and not District.startswith(DistrictName):
                                            continue;
                                        District = None;
                                        AreaElems[AreaIdx].click()
                                        time.sleep(itime)
                                        #-------------------------------------遍历每个地市下的4种资产-------------------------------------
                                        # houseTypeArr = ["住宅用房", "商业用房", "工业用房", "其他用房"];
                                        houseTypeArr = ["住宅用房"];
                                        #找到过滤条件参数块
                                        for i in range(len(houseTypeArr)):
                                            while True:
                                                PropElems = driver.find_elements(By.CSS_SELECTOR, f"div.sf-filter ul.sf-filter-row:nth-child(1) ul.condition a");
                                                PropElem = PropElems[i];
                                                #资产名称
                                                PropName = PropElem.text;
                                                # 直接到指定类型
                                                if HouseType != None and not HouseType.startswith(PropName):
                                                    break;
                                                # 匹配到一致的了改为None  后面就不判断了
                                                HouseType = None;

                                                #名称在我们需要的范围内
                                                if PropName in houseTypeArr\
                                                    and PropElem.is_enabled() and PropElem.is_displayed:
                                                    PropElem.click();
                                                #不可点击，就不处理
                                                else:
                                                    continue;
                                                
                                                #等待页面加载
                                                time.sleep(itime);
                                                
                                                #校验类型点击成功
                                                PropElem = driver.find_element(By.CSS_SELECTOR, 'div.sf-filter ul.sf-filter-row:nth-child(1) ul.condition > li > em > a.selected');
                                                if not PropElem.text.startswith(PropName):
                                                    print("PropElem is not be selected!!");
                                                    #刷新页面
                                                    driver.refresh();
                                                    time.sleep(itime);
                                                    continue;
                                                

                                                # 类型点击成功
                                                #等待网页加载
                                                driver.refresh();
                                                time.sleep(random.choice(range(itime,itime+50)));

                                                # 获取到总页数
                                                PageNo = 0;
                                                if driver.find_element(By.CSS_SELECTOR, 'em.page-total'):
                                                    PageNo = int(driver.find_element(By.CSS_SELECTOR, 'em.page-total').text)
                                                
                                                #--------------------------------------------------------------------------------------------
                                                #最多访问前50页
                                                MaxPage = 5
                                                if PageNo > MaxPage:
                                                    PageNo = MaxPage
                                                if CurrPage != None and CurrPage > MaxPage:
                                                    CurrPage = MaxPage
                                                    
                                                p = 0;
                                                while p < PageNo:
                                                    p = p + 1
                                                    if CurrPage != None and CurrPage != p:
                                                        continue
                                                    CurrPage = None;
                                                    
                                                    try:
                                                        ListUrl = driver.current_url
                                                        page_match = re.search(r'page=(\d+)', ListUrl)
                                                        if page_match:
                                                            ListUrl = ListUrl.replace('&page='+page_match.group(1), '')
                                                        
                                                        driver.get(ListUrl+'&page='+str(p));
                                                        time.sleep(itime)

                                                        #如果未进入目标页，则触发异常，等待人工处理
                                                        while 'https://sf.taobao.com/list' not in driver.current_url or '司法拍卖' not in driver.title:
                                                            print("Not LoginURL Wait!! LoginURL=", AliSFURL);
                                                            WarningPrompt(Type=EList["E101"]);
                                                            time.sleep(itime);
                                                        
                                                            loginCount = 0
                                                            while loginCount < 10:
                                                                # 登录
                                                                soup = BeautifulSoup(driver.page_source, "lxml")
                                                                loginInput = soup.select('input#fm-login-id');
                                                                if len(loginInput) > 0:
                                                                    LoginHandle(driver)            

                                                                # TODO 拉条或者登录操作
                                                                slideCount = 0
                                                                while slideCount < 13:
                                                                    time.sleep(2)
                                                                    soup = BeautifulSoup(driver.page_source, "lxml")
                                                                    sbBtn = soup.select('div.errloading')
                                                                    huakuai = soup.select('span#nc_1_n1z');

                                                                    if len(sbBtn) > 0:
                                                                        driver.find_element(By.CSS_SELECTOR, 'div.errloading').click()
                                                                        time.sleep(1)
                                                                    soup = BeautifulSoup(driver.page_source, "lxml")
                                                                    huakuai = soup.select('span#nc_1_n1z');
                                                                    
                                                                    # 出现了滑块
                                                                    if len(huakuai) > 0:
                                                                        huakuaiEle = driver.find_element(By.CSS_SELECTOR, 'span#nc_1_n1z')
                                                                        if huakuaiEle.is_displayed():
                                                                            ScrollBarHandle(driver)
                                                                            slideCount = slideCount + 1
                                                                            if slideCount % 3 == 0:
                                                                                driver.refresh()
                                                                                time.sleep(5)
                                                                        else:
                                                                            break
                                                                    else: 
                                                                        break

                                                                # 登录
                                                                soup = BeautifulSoup(driver.page_source, "lxml")
                                                                loginInput = soup.select('input#fm-login-id');
                                                                if len(loginInput) > 0:
                                                                    LoginHandle(driver)

                                                                loginCount = loginCount + 1
                                                                
                                                                soup = BeautifulSoup(driver.page_source, "lxml")
                                                                aaaa = soup.select('div.top-nav-container');
                                                                if len(aaaa) > 0:
                                                                    break

                                                        htmlcontent = driver.page_source.encode(Coder, "ignore");
                                                        #将网页进行解析，获得房屋信息列表
                                                        SFList = SFListParse(htmlcontent, PropName, ProvName, CityName);

                                                        # if driver.find_element(By.CSS_SELECTOR, 'em.page-total'):
                                                        #     PageNo = int(driver.find_element(By.CSS_SELECTOR, 'em.page-total').text)
                                                        
                                                        #将解析获得的房屋信息，保存成JSON文件
                                                        if len(SFList):
                                                            # 数据存到数据库 调用接口
                                                            PageListSave({"grabProvince": ProvName, "grabCity": CityName, "grabDistrict": DistrictName, "houseType": PropName, "grabPage": str(p), "spiderHouses": SFList});
                                                            # 状态修改为进行中
                                                            StatusModify(Status=3)
                                                            print("Ali List Success, Prov="+ProvName+",City="+CityName+",District="+DistrictName+",Type="+PropName+",Page="+str(p))
                                                        else:
                                                            # 告警
                                                            print("Parse Page SFList Null !! ");
                                                            WarningPrompt(Type=EList["E107"]);
                                                        #end for
                                                    except Exception as e:
                                                        print("Parse Page Item error!! ", e);
                                                        WarningPrompt(Type=EList["E106"]);
                                                        CheckDriverExit(driver)
                                                        
                                                    #end for PageNum
                                                    time.sleep(random.choice(range(itime,itime+50)));
                                                break;
                                    except:
                                        print("区错误......");
                                #end for Property
                                break;

                            except Exception as e:
                                print("Parse Page error!! ", e);
                                WarningPrompt(Type=EList["E103"]);
                                #检查浏览器是否可用
                                CheckDriverExit(driver)
                                time.sleep(itime);
                                continue;
                    break;
                except:
                    print("Parse City error!! ", e);
                    WarningPrompt(Type=EList["E104"]);
                    #检查浏览器是否可用
                    CheckDriverExit(driver)
                    time.sleep(itime);
                    continue;
                
            #end while 省份、地址、资产类别遍历
            break;

        except Exception as e:
            print("Parse List Error!!!!", e);
            WarningPrompt(Type=EList["E105"]);
            #检查浏览器是否可用
            CheckDriverExit(driver)
            continue;
    #end while True
    #end


def ListParseOneProv(Prov, City, District, CurrPage, HouseType, driver):
    #获得各地市的房产列表页，并插入到数据库
    AreaInfo = (Prov, None);
    GetSFList(AreaInfo, City, District, CurrPage, HouseType, driver);

# 处理省份函数
def ProvConsumer(Prov, City, District, CurrPage, HouseType, driver, SeleniumH):
    try:
        print("Consume Msg :", Prov);
        #如果浏览器关闭，则触发异常直接结束任务
        if SeleniumH.Check()==False:
            SeleniumH.Quit();
            sys.exit(1);
        #
        ListParseOneProv(Prov, City, District, CurrPage, HouseType, driver);
        #输出结果
        return 0;
    except Exception as e:
        print("queue QueueProv except info ", e)
    
# 省份解析
def ProvParse(proxyUrl, City, District, CurrPage, HouseType, serverIp):
    RabbitParam = Config["RabbitMQ"];
    #初始化Rabbit
    RabbitH = RabbitUtil(host=RabbitParam["host"], user=RabbitParam["user"], passwd=RabbitParam["passwd"], vhost=RabbitParam["vhost"]);
    #Selenium类 初始化
    SeleniumH = SeleniumUtil(ProxyInfo=PorxyInfo(proxyUrl)); #ProxyInfo=PorxyInfo());
    driver = SeleniumH.GetDriver();

    # 测试使用
    ProvConsumer('广东', City, District, CurrPage, HouseType, driver, SeleniumH)

    #等待浏览器打开
    time.sleep(2);
    # 消费队列
    def ProvQueue(Prov):
        return ProvConsumer(Prov.decode('utf-8'), City, District, CurrPage, HouseType, driver, SeleniumH)
    
    serverIp = serverIp.replace(".", "_");
    QueueName = Config["Spider"]["alilqName"]+serverIp
    RabbitH.CreateQueues([QueueName]);
    RabbitH.RegistConsumer(QueueName, ProvQueue);
    #释放资源
    SeleniumH.Quit();
    RabbitH.Close();


# 执行列表信息获取
if __name__ == "__main__":
    # 获取爬虫基本参数
    ParaMap = GetSpiderParams()
    threadNum = ParaMap["threadNum"]

    #创建2个线程的线程池
    with ThreadPoolExecutor(max_workers=threadNum) as t:
        #创建待运行线程（如果中途线程中，关掉浏览器，则直接结束任务，并触发新的任务执行）
        all_task = [ t.submit(ProvParse, ParaMap['proxyUrl'], ParaMap['grabCity'], ParaMap['grabDistrict'], ParaMap['grabPage'], ParaMap['houseType'], ParaMap['serverIp']) for _ in range(threadNum*20) ];
        #print(wait(all_task, timeout=2.5));
        #等待所有任务完成
        for future in as_completed(all_task):
            data = future.result();
            print("in main: get page {}s success".format(data));

    sys.exit(0);





